{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to DataFrames\n",
    "**[Bogumił Kamiński](http://bogumilkaminski.pl/about/), November 20, 2020**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "using DataFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "using BenchmarkTools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "using CategoricalArrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "using PooledArrays"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Performance tips"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Access by column number is faster than by name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  5.099 ns (0 allocations: 0 bytes)\n",
      "  13.239 ns (0 allocations: 0 bytes)\n"
     ]
    }
   ],
   "source": [
    "x = DataFrame(rand(5, 1000), :auto)\n",
    "@btime $x[!, 500];\n",
    "@btime $x.x500;"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### When working with data `DataFrame` use barrier functions or type annotation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  110.717 ms (5999011 allocations: 122.06 MiB)\n"
     ]
    }
   ],
   "source": [
    "using Random\n",
    "function f_bad() # this function will be slow\n",
    "    Random.seed!(1); x = DataFrame(rand(1000000,2), :auto)\n",
    "    y, z = x[!, 1], x[!, 2]\n",
    "    p = 0.0\n",
    "    for i in 1:nrow(x)\n",
    "        p += y[i]*z[i]\n",
    "    end\n",
    "    p\n",
    "end\n",
    "\n",
    "@btime f_bad();\n",
    "# if you run @code_warntype f_bad() then you notice\n",
    "# that Julia does not know column types of `DataFrame`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  9.821 ms (33 allocations: 30.52 MiB)\n",
      "  9.323 ms (33 allocations: 30.52 MiB)\n"
     ]
    }
   ],
   "source": [
    "# solution 1 is to use barrier function (it should be possible to use it in almost any code)\n",
    "function f_inner(y,z)\n",
    "   p = 0.0\n",
    "   for i in 1:length(y)\n",
    "       p += y[i]*z[i]\n",
    "   end\n",
    "   p\n",
    "end\n",
    "\n",
    "function f_barrier() # extract the work to an inner function\n",
    "    Random.seed!(1); x = DataFrame(rand(1000000,2), :auto)\n",
    "    f_inner(x[!, 1], x[!, 2])\n",
    "end\n",
    "\n",
    "using LinearAlgebra\n",
    "function f_inbuilt() # or use inbuilt function if possible\n",
    "    Random.seed!(1); x = DataFrame(rand(1000000,2), :auto)\n",
    "    dot(x[!, 1], x[!, 2])\n",
    "end\n",
    "\n",
    "@btime f_barrier();\n",
    "@btime f_inbuilt();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  9.723 ms (33 allocations: 30.52 MiB)\n"
     ]
    }
   ],
   "source": [
    "# solution 2 is to provide the types of extracted columns\n",
    "# it is simpler but there are cases in which you will not know these types\n",
    "# This example  assumes that you have DataFrames master at least from August 31, 2018\n",
    "function f_typed()\n",
    "    Random.seed!(1); x = DataFrame(rand(1000000,2), :auto)\n",
    "    y::Vector{Float64}, z::Vector{Float64} = x[!, 1], x[!, 2]\n",
    "    p = 0.0\n",
    "    for i in 1:nrow(x)\n",
    "        p += y[i]*z[i]\n",
    "    end\n",
    "    p\n",
    "end\n",
    "\n",
    "@btime f_typed();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In general for tall and narrow tables it is often useful to use `Tables.rowtable`, `Tables.columntable` or `Tables.namedtupleiterator` for intermediate processing of data in a type-stable way."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Consider using delayed `DataFrame` creation technique"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "also notice the difference in performance between copying vs non-copying data frame creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  26.882 ms (1949725 allocations: 37.42 MiB)\n",
      "  29.538 ms (1949925 allocations: 45.05 MiB)\n",
      "  3.740 ms (825 allocations: 7.68 MiB)\n",
      "  4.905 ms (1025 allocations: 15.32 MiB)\n"
     ]
    }
   ],
   "source": [
    "function f1()\n",
    "    x = DataFrame([Vector{Float64}(undef, 10^4) for i in 1:100], :auto, copycols=false) # we work with a DataFrame directly\n",
    "    for c in 1:ncol(x)\n",
    "        d = x[!, c]\n",
    "        for r in 1:nrow(x)\n",
    "            d[r] = rand()\n",
    "        end\n",
    "    end\n",
    "    x\n",
    "end\n",
    "\n",
    "function f1a()\n",
    "    x = DataFrame([Vector{Float64}(undef, 10^4) for i in 1:100], :auto) # we work with a DataFrame directly\n",
    "    for c in 1:ncol(x)\n",
    "        d = x[!, c]\n",
    "        for r in 1:nrow(x)\n",
    "            d[r] = rand()\n",
    "        end\n",
    "    end\n",
    "    x\n",
    "end\n",
    "\n",
    "function f2()\n",
    "    x = Vector{Any}(undef, 100)\n",
    "    for c in 1:length(x)\n",
    "        d = Vector{Float64}(undef, 10^4)\n",
    "        for r in 1:length(d)\n",
    "            d[r] = rand()\n",
    "        end\n",
    "        x[c] = d\n",
    "    end\n",
    "    DataFrame(x, :auto, copycols=false) # we delay creation of DataFrame after we have our job done\n",
    "end\n",
    "\n",
    "function f2a()\n",
    "    x = Vector{Any}(undef, 100)\n",
    "    for c in 1:length(x)\n",
    "        d = Vector{Float64}(undef, 10^4)\n",
    "        for r in 1:length(d)\n",
    "            d[r] = rand()\n",
    "        end\n",
    "        x[c] = d\n",
    "    end\n",
    "    DataFrame(x, :auto) # we delay creation of DataFrame after we have our job done\n",
    "end\n",
    "\n",
    "@btime f1();\n",
    "@btime f1a();\n",
    "@btime f2();\n",
    "@btime f2a();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### You can add rows to a `DataFrame` in place and it is fast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  10.179 ms (186 allocations: 38.16 MiB)\n",
      "  1.100 μs (17 allocations: 832 bytes)\n",
      "  498.964 ns (16 allocations: 256 bytes)\n"
     ]
    }
   ],
   "source": [
    "x = DataFrame(rand(10^6, 5), :auto)\n",
    "y = DataFrame(transpose(1.0:5.0), :auto)\n",
    "z = [1.0:5.0;]\n",
    "\n",
    "@btime vcat($x, $y); # creates a new DataFrame - slow\n",
    "@btime append!($x, $y); # in place - fast\n",
    "\n",
    "x = DataFrame(rand(10^6, 5), :auto) # reset to the same starting point\n",
    "@btime push!($x, $z); # add a single row in place - fast"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Allowing `missing` as well as `categorical` slows down computations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Int64\n",
      " raw:\n",
      "  4.178 ms (7 allocations: 7.63 MiB)\n",
      " categorical:\n",
      "  16.534 ms (4 allocations: 608 bytes)\n",
      "String\n",
      " raw:\n",
      "  26.217 ms (4 allocations: 608 bytes)\n",
      " categorical:\n",
      "  27.760 ms (4 allocations: 608 bytes)\n",
      "Union{Missing, Int64}\n",
      " raw:\n",
      "  9.386 ms (4 allocations: 624 bytes)\n",
      " categorical:\n",
      "  17.038 ms (4 allocations: 608 bytes)\n",
      "Union{Missing, String}\n",
      " raw:\n",
      "  42.756 ms (4 allocations: 608 bytes)\n",
      " categorical:\n",
      "  38.164 ms (4 allocations: 608 bytes)\n"
     ]
    }
   ],
   "source": [
    "using StatsBase\n",
    "\n",
    "function test(data) # uses countmap function to test performance\n",
    "    println(eltype(data))\n",
    "    x = rand(data, 10^6)\n",
    "    y = categorical(x)\n",
    "    println(\" raw:\")\n",
    "    @btime countmap($x)\n",
    "    println(\" categorical:\")\n",
    "    @btime countmap($y)\n",
    "    nothing\n",
    "end\n",
    "\n",
    "test(1:10)\n",
    "test([randstring() for i in 1:10])\n",
    "test(allowmissing(1:10))\n",
    "test(allowmissing([randstring() for i in 1:10]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### When aggregating use column selector and prefer categorical or pooled array grouping variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = DataFrame(x=rand('a':'d', 10^7), y=1);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p><b>GroupedDataFrame with 4 groups based on key: x</b></p><p><i>First Group (2498877 rows): x = 'b'</i></p><table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y</th></tr><tr><th></th><th>Char</th><th>Int64</th></tr></thead><tbody><tr><th>1</th><td>b</td><td>1</td></tr><tr><th>2</th><td>b</td><td>1</td></tr><tr><th>3</th><td>b</td><td>1</td></tr><tr><th>4</th><td>b</td><td>1</td></tr><tr><th>5</th><td>b</td><td>1</td></tr><tr><th>6</th><td>b</td><td>1</td></tr><tr><th>7</th><td>b</td><td>1</td></tr><tr><th>8</th><td>b</td><td>1</td></tr><tr><th>9</th><td>b</td><td>1</td></tr><tr><th>10</th><td>b</td><td>1</td></tr><tr><th>11</th><td>b</td><td>1</td></tr><tr><th>12</th><td>b</td><td>1</td></tr><tr><th>13</th><td>b</td><td>1</td></tr><tr><th>14</th><td>b</td><td>1</td></tr><tr><th>15</th><td>b</td><td>1</td></tr><tr><th>16</th><td>b</td><td>1</td></tr><tr><th>17</th><td>b</td><td>1</td></tr><tr><th>18</th><td>b</td><td>1</td></tr><tr><th>19</th><td>b</td><td>1</td></tr><tr><th>20</th><td>b</td><td>1</td></tr><tr><th>21</th><td>b</td><td>1</td></tr><tr><th>22</th><td>b</td><td>1</td></tr><tr><th>23</th><td>b</td><td>1</td></tr><tr><th>24</th><td>b</td><td>1</td></tr><tr><th>25</th><td>b</td><td>1</td></tr><tr><th>26</th><td>b</td><td>1</td></tr><tr><th>27</th><td>b</td><td>1</td></tr><tr><th>28</th><td>b</td><td>1</td></tr><tr><th>29</th><td>b</td><td>1</td></tr><tr><th>30</th><td>b</td><td>1</td></tr><tr><th>&vellip;</th><td>&vellip;</td><td>&vellip;</td></tr></tbody></table><p>&vellip;</p><p><i>Last Group (2501586 rows): x = 'd'</i></p><table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y</th></tr><tr><th></th><th>Char</th><th>Int64</th></tr></thead><tbody><tr><th>1</th><td>d</td><td>1</td></tr><tr><th>2</th><td>d</td><td>1</td></tr><tr><th>3</th><td>d</td><td>1</td></tr><tr><th>4</th><td>d</td><td>1</td></tr><tr><th>5</th><td>d</td><td>1</td></tr><tr><th>6</th><td>d</td><td>1</td></tr><tr><th>7</th><td>d</td><td>1</td></tr><tr><th>8</th><td>d</td><td>1</td></tr><tr><th>9</th><td>d</td><td>1</td></tr><tr><th>10</th><td>d</td><td>1</td></tr><tr><th>11</th><td>d</td><td>1</td></tr><tr><th>12</th><td>d</td><td>1</td></tr><tr><th>13</th><td>d</td><td>1</td></tr><tr><th>14</th><td>d</td><td>1</td></tr><tr><th>15</th><td>d</td><td>1</td></tr><tr><th>16</th><td>d</td><td>1</td></tr><tr><th>17</th><td>d</td><td>1</td></tr><tr><th>18</th><td>d</td><td>1</td></tr><tr><th>19</th><td>d</td><td>1</td></tr><tr><th>20</th><td>d</td><td>1</td></tr><tr><th>21</th><td>d</td><td>1</td></tr><tr><th>22</th><td>d</td><td>1</td></tr><tr><th>23</th><td>d</td><td>1</td></tr><tr><th>24</th><td>d</td><td>1</td></tr><tr><th>25</th><td>d</td><td>1</td></tr><tr><th>26</th><td>d</td><td>1</td></tr><tr><th>27</th><td>d</td><td>1</td></tr><tr><th>28</th><td>d</td><td>1</td></tr><tr><th>29</th><td>d</td><td>1</td></tr><tr><th>30</th><td>d</td><td>1</td></tr><tr><th>&vellip;</th><td>&vellip;</td><td>&vellip;</td></tr></tbody></table>"
      ],
      "text/latex": [
       "GroupedDataFrame with 4 groups based on key: x\n",
       "\n",
       "First Group (2498877 rows): x = 'b'\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& x & y\\\\\n",
       "\t\\hline\n",
       "\t& Char & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & b & 1 \\\\\n",
       "\t2 & b & 1 \\\\\n",
       "\t3 & b & 1 \\\\\n",
       "\t4 & b & 1 \\\\\n",
       "\t5 & b & 1 \\\\\n",
       "\t6 & b & 1 \\\\\n",
       "\t7 & b & 1 \\\\\n",
       "\t8 & b & 1 \\\\\n",
       "\t9 & b & 1 \\\\\n",
       "\t10 & b & 1 \\\\\n",
       "\t11 & b & 1 \\\\\n",
       "\t12 & b & 1 \\\\\n",
       "\t13 & b & 1 \\\\\n",
       "\t14 & b & 1 \\\\\n",
       "\t15 & b & 1 \\\\\n",
       "\t16 & b & 1 \\\\\n",
       "\t17 & b & 1 \\\\\n",
       "\t18 & b & 1 \\\\\n",
       "\t19 & b & 1 \\\\\n",
       "\t20 & b & 1 \\\\\n",
       "\t21 & b & 1 \\\\\n",
       "\t22 & b & 1 \\\\\n",
       "\t23 & b & 1 \\\\\n",
       "\t24 & b & 1 \\\\\n",
       "\t25 & b & 1 \\\\\n",
       "\t26 & b & 1 \\\\\n",
       "\t27 & b & 1 \\\\\n",
       "\t28 & b & 1 \\\\\n",
       "\t29 & b & 1 \\\\\n",
       "\t30 & b & 1 \\\\\n",
       "\t$\\dots$ & $\\dots$ & $\\dots$ \\\\\n",
       "\\end{tabular}\n",
       "\n",
       "$\\dots$\n",
       "\n",
       "Last Group (2501586 rows): x = 'd'\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& x & y\\\\\n",
       "\t\\hline\n",
       "\t& Char & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & d & 1 \\\\\n",
       "\t2 & d & 1 \\\\\n",
       "\t3 & d & 1 \\\\\n",
       "\t4 & d & 1 \\\\\n",
       "\t5 & d & 1 \\\\\n",
       "\t6 & d & 1 \\\\\n",
       "\t7 & d & 1 \\\\\n",
       "\t8 & d & 1 \\\\\n",
       "\t9 & d & 1 \\\\\n",
       "\t10 & d & 1 \\\\\n",
       "\t11 & d & 1 \\\\\n",
       "\t12 & d & 1 \\\\\n",
       "\t13 & d & 1 \\\\\n",
       "\t14 & d & 1 \\\\\n",
       "\t15 & d & 1 \\\\\n",
       "\t16 & d & 1 \\\\\n",
       "\t17 & d & 1 \\\\\n",
       "\t18 & d & 1 \\\\\n",
       "\t19 & d & 1 \\\\\n",
       "\t20 & d & 1 \\\\\n",
       "\t21 & d & 1 \\\\\n",
       "\t22 & d & 1 \\\\\n",
       "\t23 & d & 1 \\\\\n",
       "\t24 & d & 1 \\\\\n",
       "\t25 & d & 1 \\\\\n",
       "\t26 & d & 1 \\\\\n",
       "\t27 & d & 1 \\\\\n",
       "\t28 & d & 1 \\\\\n",
       "\t29 & d & 1 \\\\\n",
       "\t30 & d & 1 \\\\\n",
       "\t$\\dots$ & $\\dots$ & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "GroupedDataFrame with 4 groups based on key: x\n",
       "First Group (2498877 rows): x = 'b'\n",
       "\u001b[1m     Row \u001b[0m│\u001b[1m x    \u001b[0m\u001b[1m y     \u001b[0m\n",
       "\u001b[1m         \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────────┼─────────────\n",
       "       1 │ b         1\n",
       "       2 │ b         1\n",
       "       3 │ b         1\n",
       "       4 │ b         1\n",
       "       5 │ b         1\n",
       "       6 │ b         1\n",
       "       7 │ b         1\n",
       "       8 │ b         1\n",
       "       9 │ b         1\n",
       "      10 │ b         1\n",
       "      11 │ b         1\n",
       "    ⋮    │  ⋮      ⋮\n",
       " 2498867 │ b         1\n",
       " 2498868 │ b         1\n",
       " 2498869 │ b         1\n",
       " 2498870 │ b         1\n",
       " 2498871 │ b         1\n",
       " 2498872 │ b         1\n",
       " 2498873 │ b         1\n",
       " 2498874 │ b         1\n",
       " 2498875 │ b         1\n",
       " 2498876 │ b         1\n",
       " 2498877 │ b         1\n",
       "\u001b[31m   2498855 rows omitted\u001b[0m\n",
       "⋮\n",
       "Last Group (2501586 rows): x = 'd'\n",
       "\u001b[1m     Row \u001b[0m│\u001b[1m x    \u001b[0m\u001b[1m y     \u001b[0m\n",
       "\u001b[1m         \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────────┼─────────────\n",
       "       1 │ d         1\n",
       "       2 │ d         1\n",
       "       3 │ d         1\n",
       "       4 │ d         1\n",
       "       5 │ d         1\n",
       "       6 │ d         1\n",
       "       7 │ d         1\n",
       "       8 │ d         1\n",
       "       9 │ d         1\n",
       "      10 │ d         1\n",
       "      11 │ d         1\n",
       "    ⋮    │  ⋮      ⋮\n",
       " 2501576 │ d         1\n",
       " 2501577 │ d         1\n",
       " 2501578 │ d         1\n",
       " 2501579 │ d         1\n",
       " 2501580 │ d         1\n",
       " 2501581 │ d         1\n",
       " 2501582 │ d         1\n",
       " 2501583 │ d         1\n",
       " 2501584 │ d         1\n",
       " 2501585 │ d         1\n",
       " 2501586 │ d         1\n",
       "\u001b[31m   2501564 rows omitted\u001b[0m"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf = groupby(df, :x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  50.879 ms (217 allocations: 76.31 MiB)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>x1</th></tr><tr><th></th><th>Char</th><th>Int64</th></tr></thead><tbody><p>4 rows × 2 columns</p><tr><th>1</th><td>b</td><td>2498877</td></tr><tr><th>2</th><td>a</td><td>2499759</td></tr><tr><th>3</th><td>c</td><td>2499778</td></tr><tr><th>4</th><td>d</td><td>2501586</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& x & x1\\\\\n",
       "\t\\hline\n",
       "\t& Char & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & b & 2498877 \\\\\n",
       "\t2 & a & 2499759 \\\\\n",
       "\t3 & c & 2499778 \\\\\n",
       "\t4 & d & 2501586 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m4×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x    \u001b[0m\u001b[1m x1      \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Int64   \u001b[0m\n",
       "─────┼───────────────\n",
       "   1 │ b     2498877\n",
       "   2 │ a     2499759\n",
       "   3 │ c     2499778\n",
       "   4 │ d     2501586"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@btime combine(v -> sum(v.y), $gdf) # traditional syntax, slow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  9.445 ms (151 allocations: 11.92 KiB)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y_sum</th></tr><tr><th></th><th>Char</th><th>Int64</th></tr></thead><tbody><p>4 rows × 2 columns</p><tr><th>1</th><td>b</td><td>2498877</td></tr><tr><th>2</th><td>a</td><td>2499759</td></tr><tr><th>3</th><td>c</td><td>2499778</td></tr><tr><th>4</th><td>d</td><td>2501586</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& x & y\\_sum\\\\\n",
       "\t\\hline\n",
       "\t& Char & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & b & 2498877 \\\\\n",
       "\t2 & a & 2499759 \\\\\n",
       "\t3 & c & 2499778 \\\\\n",
       "\t4 & d & 2501586 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m4×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x    \u001b[0m\u001b[1m y_sum   \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Int64   \u001b[0m\n",
       "─────┼───────────────\n",
       "   1 │ b     2498877\n",
       "   2 │ a     2499759\n",
       "   3 │ c     2499778\n",
       "   4 │ d     2501586"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@btime combine($gdf, :y=>sum) # use column selector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "transform!(df, :x => categorical => :x);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p><b>GroupedDataFrame with 4 groups based on key: x</b></p><p><i>First Group (2499759 rows): x = CategoricalValue{Char,UInt32} 'a'</i></p><table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y</th></tr><tr><th></th><th>Cat…</th><th>Int64</th></tr></thead><tbody><tr><th>1</th><td>a</td><td>1</td></tr><tr><th>2</th><td>a</td><td>1</td></tr><tr><th>3</th><td>a</td><td>1</td></tr><tr><th>4</th><td>a</td><td>1</td></tr><tr><th>5</th><td>a</td><td>1</td></tr><tr><th>6</th><td>a</td><td>1</td></tr><tr><th>7</th><td>a</td><td>1</td></tr><tr><th>8</th><td>a</td><td>1</td></tr><tr><th>9</th><td>a</td><td>1</td></tr><tr><th>10</th><td>a</td><td>1</td></tr><tr><th>11</th><td>a</td><td>1</td></tr><tr><th>12</th><td>a</td><td>1</td></tr><tr><th>13</th><td>a</td><td>1</td></tr><tr><th>14</th><td>a</td><td>1</td></tr><tr><th>15</th><td>a</td><td>1</td></tr><tr><th>16</th><td>a</td><td>1</td></tr><tr><th>17</th><td>a</td><td>1</td></tr><tr><th>18</th><td>a</td><td>1</td></tr><tr><th>19</th><td>a</td><td>1</td></tr><tr><th>20</th><td>a</td><td>1</td></tr><tr><th>21</th><td>a</td><td>1</td></tr><tr><th>22</th><td>a</td><td>1</td></tr><tr><th>23</th><td>a</td><td>1</td></tr><tr><th>24</th><td>a</td><td>1</td></tr><tr><th>25</th><td>a</td><td>1</td></tr><tr><th>26</th><td>a</td><td>1</td></tr><tr><th>27</th><td>a</td><td>1</td></tr><tr><th>28</th><td>a</td><td>1</td></tr><tr><th>29</th><td>a</td><td>1</td></tr><tr><th>30</th><td>a</td><td>1</td></tr><tr><th>&vellip;</th><td>&vellip;</td><td>&vellip;</td></tr></tbody></table><p>&vellip;</p><p><i>Last Group (2501586 rows): x = CategoricalValue{Char,UInt32} 'd'</i></p><table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y</th></tr><tr><th></th><th>Cat…</th><th>Int64</th></tr></thead><tbody><tr><th>1</th><td>d</td><td>1</td></tr><tr><th>2</th><td>d</td><td>1</td></tr><tr><th>3</th><td>d</td><td>1</td></tr><tr><th>4</th><td>d</td><td>1</td></tr><tr><th>5</th><td>d</td><td>1</td></tr><tr><th>6</th><td>d</td><td>1</td></tr><tr><th>7</th><td>d</td><td>1</td></tr><tr><th>8</th><td>d</td><td>1</td></tr><tr><th>9</th><td>d</td><td>1</td></tr><tr><th>10</th><td>d</td><td>1</td></tr><tr><th>11</th><td>d</td><td>1</td></tr><tr><th>12</th><td>d</td><td>1</td></tr><tr><th>13</th><td>d</td><td>1</td></tr><tr><th>14</th><td>d</td><td>1</td></tr><tr><th>15</th><td>d</td><td>1</td></tr><tr><th>16</th><td>d</td><td>1</td></tr><tr><th>17</th><td>d</td><td>1</td></tr><tr><th>18</th><td>d</td><td>1</td></tr><tr><th>19</th><td>d</td><td>1</td></tr><tr><th>20</th><td>d</td><td>1</td></tr><tr><th>21</th><td>d</td><td>1</td></tr><tr><th>22</th><td>d</td><td>1</td></tr><tr><th>23</th><td>d</td><td>1</td></tr><tr><th>24</th><td>d</td><td>1</td></tr><tr><th>25</th><td>d</td><td>1</td></tr><tr><th>26</th><td>d</td><td>1</td></tr><tr><th>27</th><td>d</td><td>1</td></tr><tr><th>28</th><td>d</td><td>1</td></tr><tr><th>29</th><td>d</td><td>1</td></tr><tr><th>30</th><td>d</td><td>1</td></tr><tr><th>&vellip;</th><td>&vellip;</td><td>&vellip;</td></tr></tbody></table>"
      ],
      "text/latex": [
       "GroupedDataFrame with 4 groups based on key: x\n",
       "\n",
       "First Group (2499759 rows): x = CategoricalValue\\{Char,UInt32\\} 'a'\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& x & y\\\\\n",
       "\t\\hline\n",
       "\t& Cat… & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & a & 1 \\\\\n",
       "\t2 & a & 1 \\\\\n",
       "\t3 & a & 1 \\\\\n",
       "\t4 & a & 1 \\\\\n",
       "\t5 & a & 1 \\\\\n",
       "\t6 & a & 1 \\\\\n",
       "\t7 & a & 1 \\\\\n",
       "\t8 & a & 1 \\\\\n",
       "\t9 & a & 1 \\\\\n",
       "\t10 & a & 1 \\\\\n",
       "\t11 & a & 1 \\\\\n",
       "\t12 & a & 1 \\\\\n",
       "\t13 & a & 1 \\\\\n",
       "\t14 & a & 1 \\\\\n",
       "\t15 & a & 1 \\\\\n",
       "\t16 & a & 1 \\\\\n",
       "\t17 & a & 1 \\\\\n",
       "\t18 & a & 1 \\\\\n",
       "\t19 & a & 1 \\\\\n",
       "\t20 & a & 1 \\\\\n",
       "\t21 & a & 1 \\\\\n",
       "\t22 & a & 1 \\\\\n",
       "\t23 & a & 1 \\\\\n",
       "\t24 & a & 1 \\\\\n",
       "\t25 & a & 1 \\\\\n",
       "\t26 & a & 1 \\\\\n",
       "\t27 & a & 1 \\\\\n",
       "\t28 & a & 1 \\\\\n",
       "\t29 & a & 1 \\\\\n",
       "\t30 & a & 1 \\\\\n",
       "\t$\\dots$ & $\\dots$ & $\\dots$ \\\\\n",
       "\\end{tabular}\n",
       "\n",
       "$\\dots$\n",
       "\n",
       "Last Group (2501586 rows): x = CategoricalValue\\{Char,UInt32\\} 'd'\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& x & y\\\\\n",
       "\t\\hline\n",
       "\t& Cat… & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & d & 1 \\\\\n",
       "\t2 & d & 1 \\\\\n",
       "\t3 & d & 1 \\\\\n",
       "\t4 & d & 1 \\\\\n",
       "\t5 & d & 1 \\\\\n",
       "\t6 & d & 1 \\\\\n",
       "\t7 & d & 1 \\\\\n",
       "\t8 & d & 1 \\\\\n",
       "\t9 & d & 1 \\\\\n",
       "\t10 & d & 1 \\\\\n",
       "\t11 & d & 1 \\\\\n",
       "\t12 & d & 1 \\\\\n",
       "\t13 & d & 1 \\\\\n",
       "\t14 & d & 1 \\\\\n",
       "\t15 & d & 1 \\\\\n",
       "\t16 & d & 1 \\\\\n",
       "\t17 & d & 1 \\\\\n",
       "\t18 & d & 1 \\\\\n",
       "\t19 & d & 1 \\\\\n",
       "\t20 & d & 1 \\\\\n",
       "\t21 & d & 1 \\\\\n",
       "\t22 & d & 1 \\\\\n",
       "\t23 & d & 1 \\\\\n",
       "\t24 & d & 1 \\\\\n",
       "\t25 & d & 1 \\\\\n",
       "\t26 & d & 1 \\\\\n",
       "\t27 & d & 1 \\\\\n",
       "\t28 & d & 1 \\\\\n",
       "\t29 & d & 1 \\\\\n",
       "\t30 & d & 1 \\\\\n",
       "\t$\\dots$ & $\\dots$ & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "GroupedDataFrame with 4 groups based on key: x\n",
       "First Group (2499759 rows): x = CategoricalValue{Char,UInt32} 'a'\n",
       "\u001b[1m     Row \u001b[0m│\u001b[1m x    \u001b[0m\u001b[1m y     \u001b[0m\n",
       "\u001b[1m         \u001b[0m│\u001b[90m Cat… \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────────┼─────────────\n",
       "       1 │ a         1\n",
       "       2 │ a         1\n",
       "       3 │ a         1\n",
       "       4 │ a         1\n",
       "       5 │ a         1\n",
       "       6 │ a         1\n",
       "       7 │ a         1\n",
       "       8 │ a         1\n",
       "       9 │ a         1\n",
       "      10 │ a         1\n",
       "      11 │ a         1\n",
       "    ⋮    │  ⋮      ⋮\n",
       " 2499749 │ a         1\n",
       " 2499750 │ a         1\n",
       " 2499751 │ a         1\n",
       " 2499752 │ a         1\n",
       " 2499753 │ a         1\n",
       " 2499754 │ a         1\n",
       " 2499755 │ a         1\n",
       " 2499756 │ a         1\n",
       " 2499757 │ a         1\n",
       " 2499758 │ a         1\n",
       " 2499759 │ a         1\n",
       "\u001b[31m   2499737 rows omitted\u001b[0m\n",
       "⋮\n",
       "Last Group (2501586 rows): x = CategoricalValue{Char,UInt32} 'd'\n",
       "\u001b[1m     Row \u001b[0m│\u001b[1m x    \u001b[0m\u001b[1m y     \u001b[0m\n",
       "\u001b[1m         \u001b[0m│\u001b[90m Cat… \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────────┼─────────────\n",
       "       1 │ d         1\n",
       "       2 │ d         1\n",
       "       3 │ d         1\n",
       "       4 │ d         1\n",
       "       5 │ d         1\n",
       "       6 │ d         1\n",
       "       7 │ d         1\n",
       "       8 │ d         1\n",
       "       9 │ d         1\n",
       "      10 │ d         1\n",
       "      11 │ d         1\n",
       "    ⋮    │  ⋮      ⋮\n",
       " 2501576 │ d         1\n",
       " 2501577 │ d         1\n",
       " 2501578 │ d         1\n",
       " 2501579 │ d         1\n",
       " 2501580 │ d         1\n",
       " 2501581 │ d         1\n",
       " 2501582 │ d         1\n",
       " 2501583 │ d         1\n",
       " 2501584 │ d         1\n",
       " 2501585 │ d         1\n",
       " 2501586 │ d         1\n",
       "\u001b[31m   2501564 rows omitted\u001b[0m"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf = groupby(df, :x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  9.252 ms (163 allocations: 12.80 KiB)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y_sum</th></tr><tr><th></th><th>Cat…</th><th>Int64</th></tr></thead><tbody><p>4 rows × 2 columns</p><tr><th>1</th><td>a</td><td>2499759</td></tr><tr><th>2</th><td>b</td><td>2498877</td></tr><tr><th>3</th><td>c</td><td>2499778</td></tr><tr><th>4</th><td>d</td><td>2501586</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& x & y\\_sum\\\\\n",
       "\t\\hline\n",
       "\t& Cat… & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & a & 2499759 \\\\\n",
       "\t2 & b & 2498877 \\\\\n",
       "\t3 & c & 2499778 \\\\\n",
       "\t4 & d & 2501586 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m4×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x    \u001b[0m\u001b[1m y_sum   \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Cat… \u001b[0m\u001b[90m Int64   \u001b[0m\n",
       "─────┼───────────────\n",
       "   1 │ a     2499759\n",
       "   2 │ b     2498877\n",
       "   3 │ c     2499778\n",
       "   4 │ d     2501586"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@btime combine($gdf, :y=>sum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y</th></tr><tr><th></th><th>Char</th><th>Int64</th></tr></thead><tbody><p>10,000,000 rows × 2 columns</p><tr><th>1</th><td>b</td><td>1</td></tr><tr><th>2</th><td>b</td><td>1</td></tr><tr><th>3</th><td>a</td><td>1</td></tr><tr><th>4</th><td>b</td><td>1</td></tr><tr><th>5</th><td>a</td><td>1</td></tr><tr><th>6</th><td>a</td><td>1</td></tr><tr><th>7</th><td>c</td><td>1</td></tr><tr><th>8</th><td>a</td><td>1</td></tr><tr><th>9</th><td>d</td><td>1</td></tr><tr><th>10</th><td>d</td><td>1</td></tr><tr><th>11</th><td>a</td><td>1</td></tr><tr><th>12</th><td>a</td><td>1</td></tr><tr><th>13</th><td>a</td><td>1</td></tr><tr><th>14</th><td>b</td><td>1</td></tr><tr><th>15</th><td>a</td><td>1</td></tr><tr><th>16</th><td>a</td><td>1</td></tr><tr><th>17</th><td>b</td><td>1</td></tr><tr><th>18</th><td>d</td><td>1</td></tr><tr><th>19</th><td>b</td><td>1</td></tr><tr><th>20</th><td>a</td><td>1</td></tr><tr><th>21</th><td>d</td><td>1</td></tr><tr><th>22</th><td>c</td><td>1</td></tr><tr><th>23</th><td>c</td><td>1</td></tr><tr><th>24</th><td>b</td><td>1</td></tr><tr><th>25</th><td>b</td><td>1</td></tr><tr><th>26</th><td>d</td><td>1</td></tr><tr><th>27</th><td>a</td><td>1</td></tr><tr><th>28</th><td>d</td><td>1</td></tr><tr><th>29</th><td>d</td><td>1</td></tr><tr><th>30</th><td>a</td><td>1</td></tr><tr><th>&vellip;</th><td>&vellip;</td><td>&vellip;</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& x & y\\\\\n",
       "\t\\hline\n",
       "\t& Char & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & b & 1 \\\\\n",
       "\t2 & b & 1 \\\\\n",
       "\t3 & a & 1 \\\\\n",
       "\t4 & b & 1 \\\\\n",
       "\t5 & a & 1 \\\\\n",
       "\t6 & a & 1 \\\\\n",
       "\t7 & c & 1 \\\\\n",
       "\t8 & a & 1 \\\\\n",
       "\t9 & d & 1 \\\\\n",
       "\t10 & d & 1 \\\\\n",
       "\t11 & a & 1 \\\\\n",
       "\t12 & a & 1 \\\\\n",
       "\t13 & a & 1 \\\\\n",
       "\t14 & b & 1 \\\\\n",
       "\t15 & a & 1 \\\\\n",
       "\t16 & a & 1 \\\\\n",
       "\t17 & b & 1 \\\\\n",
       "\t18 & d & 1 \\\\\n",
       "\t19 & b & 1 \\\\\n",
       "\t20 & a & 1 \\\\\n",
       "\t21 & d & 1 \\\\\n",
       "\t22 & c & 1 \\\\\n",
       "\t23 & c & 1 \\\\\n",
       "\t24 & b & 1 \\\\\n",
       "\t25 & b & 1 \\\\\n",
       "\t26 & d & 1 \\\\\n",
       "\t27 & a & 1 \\\\\n",
       "\t28 & d & 1 \\\\\n",
       "\t29 & d & 1 \\\\\n",
       "\t30 & a & 1 \\\\\n",
       "\t$\\dots$ & $\\dots$ & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m10000000×2 DataFrame\u001b[0m\n",
       "\u001b[1m      Row \u001b[0m│\u001b[1m x    \u001b[0m\u001b[1m y     \u001b[0m\n",
       "\u001b[1m          \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "──────────┼─────────────\n",
       "        1 │ b         1\n",
       "        2 │ b         1\n",
       "        3 │ a         1\n",
       "        4 │ b         1\n",
       "        5 │ a         1\n",
       "        6 │ a         1\n",
       "        7 │ c         1\n",
       "        8 │ a         1\n",
       "        9 │ d         1\n",
       "       10 │ d         1\n",
       "       11 │ a         1\n",
       "    ⋮     │  ⋮      ⋮\n",
       "  9999991 │ a         1\n",
       "  9999992 │ c         1\n",
       "  9999993 │ b         1\n",
       "  9999994 │ a         1\n",
       "  9999995 │ a         1\n",
       "  9999996 │ a         1\n",
       "  9999997 │ c         1\n",
       "  9999998 │ d         1\n",
       "  9999999 │ c         1\n",
       " 10000000 │ a         1\n",
       "\u001b[31m    9999979 rows omitted\u001b[0m"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transform!(df, :x => PooledArray{Char} => :x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p><b>GroupedDataFrame with 4 groups based on key: x</b></p><p><i>First Group (2498877 rows): x = 'b'</i></p><table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y</th></tr><tr><th></th><th>Char</th><th>Int64</th></tr></thead><tbody><tr><th>1</th><td>b</td><td>1</td></tr><tr><th>2</th><td>b</td><td>1</td></tr><tr><th>3</th><td>b</td><td>1</td></tr><tr><th>4</th><td>b</td><td>1</td></tr><tr><th>5</th><td>b</td><td>1</td></tr><tr><th>6</th><td>b</td><td>1</td></tr><tr><th>7</th><td>b</td><td>1</td></tr><tr><th>8</th><td>b</td><td>1</td></tr><tr><th>9</th><td>b</td><td>1</td></tr><tr><th>10</th><td>b</td><td>1</td></tr><tr><th>11</th><td>b</td><td>1</td></tr><tr><th>12</th><td>b</td><td>1</td></tr><tr><th>13</th><td>b</td><td>1</td></tr><tr><th>14</th><td>b</td><td>1</td></tr><tr><th>15</th><td>b</td><td>1</td></tr><tr><th>16</th><td>b</td><td>1</td></tr><tr><th>17</th><td>b</td><td>1</td></tr><tr><th>18</th><td>b</td><td>1</td></tr><tr><th>19</th><td>b</td><td>1</td></tr><tr><th>20</th><td>b</td><td>1</td></tr><tr><th>21</th><td>b</td><td>1</td></tr><tr><th>22</th><td>b</td><td>1</td></tr><tr><th>23</th><td>b</td><td>1</td></tr><tr><th>24</th><td>b</td><td>1</td></tr><tr><th>25</th><td>b</td><td>1</td></tr><tr><th>26</th><td>b</td><td>1</td></tr><tr><th>27</th><td>b</td><td>1</td></tr><tr><th>28</th><td>b</td><td>1</td></tr><tr><th>29</th><td>b</td><td>1</td></tr><tr><th>30</th><td>b</td><td>1</td></tr><tr><th>&vellip;</th><td>&vellip;</td><td>&vellip;</td></tr></tbody></table><p>&vellip;</p><p><i>Last Group (2501586 rows): x = 'd'</i></p><table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y</th></tr><tr><th></th><th>Char</th><th>Int64</th></tr></thead><tbody><tr><th>1</th><td>d</td><td>1</td></tr><tr><th>2</th><td>d</td><td>1</td></tr><tr><th>3</th><td>d</td><td>1</td></tr><tr><th>4</th><td>d</td><td>1</td></tr><tr><th>5</th><td>d</td><td>1</td></tr><tr><th>6</th><td>d</td><td>1</td></tr><tr><th>7</th><td>d</td><td>1</td></tr><tr><th>8</th><td>d</td><td>1</td></tr><tr><th>9</th><td>d</td><td>1</td></tr><tr><th>10</th><td>d</td><td>1</td></tr><tr><th>11</th><td>d</td><td>1</td></tr><tr><th>12</th><td>d</td><td>1</td></tr><tr><th>13</th><td>d</td><td>1</td></tr><tr><th>14</th><td>d</td><td>1</td></tr><tr><th>15</th><td>d</td><td>1</td></tr><tr><th>16</th><td>d</td><td>1</td></tr><tr><th>17</th><td>d</td><td>1</td></tr><tr><th>18</th><td>d</td><td>1</td></tr><tr><th>19</th><td>d</td><td>1</td></tr><tr><th>20</th><td>d</td><td>1</td></tr><tr><th>21</th><td>d</td><td>1</td></tr><tr><th>22</th><td>d</td><td>1</td></tr><tr><th>23</th><td>d</td><td>1</td></tr><tr><th>24</th><td>d</td><td>1</td></tr><tr><th>25</th><td>d</td><td>1</td></tr><tr><th>26</th><td>d</td><td>1</td></tr><tr><th>27</th><td>d</td><td>1</td></tr><tr><th>28</th><td>d</td><td>1</td></tr><tr><th>29</th><td>d</td><td>1</td></tr><tr><th>30</th><td>d</td><td>1</td></tr><tr><th>&vellip;</th><td>&vellip;</td><td>&vellip;</td></tr></tbody></table>"
      ],
      "text/latex": [
       "GroupedDataFrame with 4 groups based on key: x\n",
       "\n",
       "First Group (2498877 rows): x = 'b'\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& x & y\\\\\n",
       "\t\\hline\n",
       "\t& Char & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & b & 1 \\\\\n",
       "\t2 & b & 1 \\\\\n",
       "\t3 & b & 1 \\\\\n",
       "\t4 & b & 1 \\\\\n",
       "\t5 & b & 1 \\\\\n",
       "\t6 & b & 1 \\\\\n",
       "\t7 & b & 1 \\\\\n",
       "\t8 & b & 1 \\\\\n",
       "\t9 & b & 1 \\\\\n",
       "\t10 & b & 1 \\\\\n",
       "\t11 & b & 1 \\\\\n",
       "\t12 & b & 1 \\\\\n",
       "\t13 & b & 1 \\\\\n",
       "\t14 & b & 1 \\\\\n",
       "\t15 & b & 1 \\\\\n",
       "\t16 & b & 1 \\\\\n",
       "\t17 & b & 1 \\\\\n",
       "\t18 & b & 1 \\\\\n",
       "\t19 & b & 1 \\\\\n",
       "\t20 & b & 1 \\\\\n",
       "\t21 & b & 1 \\\\\n",
       "\t22 & b & 1 \\\\\n",
       "\t23 & b & 1 \\\\\n",
       "\t24 & b & 1 \\\\\n",
       "\t25 & b & 1 \\\\\n",
       "\t26 & b & 1 \\\\\n",
       "\t27 & b & 1 \\\\\n",
       "\t28 & b & 1 \\\\\n",
       "\t29 & b & 1 \\\\\n",
       "\t30 & b & 1 \\\\\n",
       "\t$\\dots$ & $\\dots$ & $\\dots$ \\\\\n",
       "\\end{tabular}\n",
       "\n",
       "$\\dots$\n",
       "\n",
       "Last Group (2501586 rows): x = 'd'\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& x & y\\\\\n",
       "\t\\hline\n",
       "\t& Char & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & d & 1 \\\\\n",
       "\t2 & d & 1 \\\\\n",
       "\t3 & d & 1 \\\\\n",
       "\t4 & d & 1 \\\\\n",
       "\t5 & d & 1 \\\\\n",
       "\t6 & d & 1 \\\\\n",
       "\t7 & d & 1 \\\\\n",
       "\t8 & d & 1 \\\\\n",
       "\t9 & d & 1 \\\\\n",
       "\t10 & d & 1 \\\\\n",
       "\t11 & d & 1 \\\\\n",
       "\t12 & d & 1 \\\\\n",
       "\t13 & d & 1 \\\\\n",
       "\t14 & d & 1 \\\\\n",
       "\t15 & d & 1 \\\\\n",
       "\t16 & d & 1 \\\\\n",
       "\t17 & d & 1 \\\\\n",
       "\t18 & d & 1 \\\\\n",
       "\t19 & d & 1 \\\\\n",
       "\t20 & d & 1 \\\\\n",
       "\t21 & d & 1 \\\\\n",
       "\t22 & d & 1 \\\\\n",
       "\t23 & d & 1 \\\\\n",
       "\t24 & d & 1 \\\\\n",
       "\t25 & d & 1 \\\\\n",
       "\t26 & d & 1 \\\\\n",
       "\t27 & d & 1 \\\\\n",
       "\t28 & d & 1 \\\\\n",
       "\t29 & d & 1 \\\\\n",
       "\t30 & d & 1 \\\\\n",
       "\t$\\dots$ & $\\dots$ & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "GroupedDataFrame with 4 groups based on key: x\n",
       "First Group (2498877 rows): x = 'b'\n",
       "\u001b[1m     Row \u001b[0m│\u001b[1m x    \u001b[0m\u001b[1m y     \u001b[0m\n",
       "\u001b[1m         \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────────┼─────────────\n",
       "       1 │ b         1\n",
       "       2 │ b         1\n",
       "       3 │ b         1\n",
       "       4 │ b         1\n",
       "       5 │ b         1\n",
       "       6 │ b         1\n",
       "       7 │ b         1\n",
       "       8 │ b         1\n",
       "       9 │ b         1\n",
       "      10 │ b         1\n",
       "      11 │ b         1\n",
       "    ⋮    │  ⋮      ⋮\n",
       " 2498867 │ b         1\n",
       " 2498868 │ b         1\n",
       " 2498869 │ b         1\n",
       " 2498870 │ b         1\n",
       " 2498871 │ b         1\n",
       " 2498872 │ b         1\n",
       " 2498873 │ b         1\n",
       " 2498874 │ b         1\n",
       " 2498875 │ b         1\n",
       " 2498876 │ b         1\n",
       " 2498877 │ b         1\n",
       "\u001b[31m   2498855 rows omitted\u001b[0m\n",
       "⋮\n",
       "Last Group (2501586 rows): x = 'd'\n",
       "\u001b[1m     Row \u001b[0m│\u001b[1m x    \u001b[0m\u001b[1m y     \u001b[0m\n",
       "\u001b[1m         \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────────┼─────────────\n",
       "       1 │ d         1\n",
       "       2 │ d         1\n",
       "       3 │ d         1\n",
       "       4 │ d         1\n",
       "       5 │ d         1\n",
       "       6 │ d         1\n",
       "       7 │ d         1\n",
       "       8 │ d         1\n",
       "       9 │ d         1\n",
       "      10 │ d         1\n",
       "      11 │ d         1\n",
       "    ⋮    │  ⋮      ⋮\n",
       " 2501576 │ d         1\n",
       " 2501577 │ d         1\n",
       " 2501578 │ d         1\n",
       " 2501579 │ d         1\n",
       " 2501580 │ d         1\n",
       " 2501581 │ d         1\n",
       " 2501582 │ d         1\n",
       " 2501583 │ d         1\n",
       " 2501584 │ d         1\n",
       " 2501585 │ d         1\n",
       " 2501586 │ d         1\n",
       "\u001b[31m   2501564 rows omitted\u001b[0m"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf = groupby(df, :x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  9.344 ms (158 allocations: 12.50 KiB)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y_sum</th></tr><tr><th></th><th>Char</th><th>Int64</th></tr></thead><tbody><p>4 rows × 2 columns</p><tr><th>1</th><td>b</td><td>2498877</td></tr><tr><th>2</th><td>a</td><td>2499759</td></tr><tr><th>3</th><td>c</td><td>2499778</td></tr><tr><th>4</th><td>d</td><td>2501586</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& x & y\\_sum\\\\\n",
       "\t\\hline\n",
       "\t& Char & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & b & 2498877 \\\\\n",
       "\t2 & a & 2499759 \\\\\n",
       "\t3 & c & 2499778 \\\\\n",
       "\t4 & d & 2501586 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m4×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x    \u001b[0m\u001b[1m y_sum   \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Int64   \u001b[0m\n",
       "─────┼───────────────\n",
       "   1 │ b     2498877\n",
       "   2 │ a     2499759\n",
       "   3 │ c     2499778\n",
       "   4 │ d     2501586"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@btime combine($gdf, :y=>sum)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use views instead of materializing a new DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th><th>x6</th><th>x7</th></tr><tr><th></th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th></tr></thead><tbody><p>100 rows × 1,000 columns (omitted printing of 993 columns)</p><tr><th>1</th><td>0.654616</td><td>0.719149</td><td>0.152413</td><td>0.149021</td><td>0.906955</td><td>0.601046</td><td>0.659532</td></tr><tr><th>2</th><td>0.200366</td><td>0.77317</td><td>0.54823</td><td>0.619689</td><td>0.311176</td><td>0.348622</td><td>0.116158</td></tr><tr><th>3</th><td>0.0707588</td><td>0.282814</td><td>0.809332</td><td>0.553294</td><td>0.6751</td><td>0.187457</td><td>0.640539</td></tr><tr><th>4</th><td>0.263277</td><td>0.974213</td><td>0.469272</td><td>0.976958</td><td>0.862772</td><td>0.938512</td><td>0.617304</td></tr><tr><th>5</th><td>0.58163</td><td>0.949869</td><td>0.0661904</td><td>0.863933</td><td>0.685727</td><td>0.395609</td><td>0.311059</td></tr><tr><th>6</th><td>0.616957</td><td>0.809582</td><td>0.667691</td><td>0.0241465</td><td>0.437346</td><td>0.833641</td><td>0.11238</td></tr><tr><th>7</th><td>0.321904</td><td>0.569951</td><td>0.581407</td><td>0.520908</td><td>0.34152</td><td>0.570203</td><td>0.269444</td></tr><tr><th>8</th><td>0.862673</td><td>0.992583</td><td>0.153743</td><td>0.641501</td><td>0.479036</td><td>0.858642</td><td>0.0335735</td></tr><tr><th>9</th><td>0.974797</td><td>0.185455</td><td>0.183458</td><td>0.793523</td><td>0.525708</td><td>0.263889</td><td>0.572278</td></tr><tr><th>10</th><td>0.573963</td><td>0.195312</td><td>0.483336</td><td>0.268587</td><td>0.193243</td><td>0.2797</td><td>0.265947</td></tr><tr><th>11</th><td>0.799476</td><td>0.509579</td><td>0.667967</td><td>0.0499318</td><td>0.714211</td><td>0.977774</td><td>0.588332</td></tr><tr><th>12</th><td>0.423771</td><td>0.280858</td><td>0.162556</td><td>0.629407</td><td>0.41928</td><td>0.451621</td><td>0.984615</td></tr><tr><th>13</th><td>0.0616925</td><td>0.611666</td><td>0.884873</td><td>0.312924</td><td>0.303788</td><td>0.822353</td><td>0.0804944</td></tr><tr><th>14</th><td>0.880249</td><td>0.256299</td><td>0.925109</td><td>0.13784</td><td>0.499931</td><td>0.760433</td><td>0.538864</td></tr><tr><th>15</th><td>0.162932</td><td>0.249386</td><td>0.763224</td><td>0.891417</td><td>0.443553</td><td>0.771456</td><td>0.117484</td></tr><tr><th>16</th><td>0.882158</td><td>0.644417</td><td>0.22389</td><td>0.781504</td><td>0.58965</td><td>0.148809</td><td>0.46363</td></tr><tr><th>17</th><td>0.710762</td><td>0.299941</td><td>0.521807</td><td>0.956344</td><td>0.412978</td><td>0.0570247</td><td>0.435834</td></tr><tr><th>18</th><td>0.351941</td><td>0.591889</td><td>0.0850421</td><td>0.619161</td><td>0.739657</td><td>0.800522</td><td>0.91802</td></tr><tr><th>19</th><td>0.518158</td><td>0.870301</td><td>0.523169</td><td>0.919283</td><td>0.780809</td><td>0.923941</td><td>0.962397</td></tr><tr><th>20</th><td>0.305017</td><td>0.137517</td><td>0.0850551</td><td>0.0845877</td><td>0.436503</td><td>0.309409</td><td>0.620065</td></tr><tr><th>21</th><td>0.776885</td><td>0.151316</td><td>0.663058</td><td>0.00522126</td><td>0.22959</td><td>0.61607</td><td>0.815048</td></tr><tr><th>22</th><td>0.567911</td><td>0.318317</td><td>0.942034</td><td>0.859492</td><td>0.882784</td><td>0.00790537</td><td>0.870021</td></tr><tr><th>23</th><td>0.942238</td><td>0.563475</td><td>0.510555</td><td>0.544423</td><td>0.714952</td><td>0.816133</td><td>0.515208</td></tr><tr><th>24</th><td>0.588218</td><td>0.776325</td><td>0.356093</td><td>0.334207</td><td>0.576845</td><td>0.788708</td><td>0.0206834</td></tr><tr><th>25</th><td>0.204753</td><td>0.0273146</td><td>0.157649</td><td>0.171963</td><td>0.891935</td><td>0.0866493</td><td>0.0147336</td></tr><tr><th>26</th><td>0.968665</td><td>0.526432</td><td>0.00336494</td><td>0.762573</td><td>0.736109</td><td>0.599631</td><td>0.00136439</td></tr><tr><th>27</th><td>0.101632</td><td>0.0437408</td><td>0.480526</td><td>0.178961</td><td>0.614065</td><td>0.296311</td><td>0.9397</td></tr><tr><th>28</th><td>0.385756</td><td>0.23059</td><td>0.0843208</td><td>0.248933</td><td>0.888177</td><td>0.623427</td><td>0.183932</td></tr><tr><th>29</th><td>0.285264</td><td>0.350805</td><td>0.494201</td><td>0.755728</td><td>0.293184</td><td>0.36395</td><td>0.608347</td></tr><tr><th>30</th><td>0.724326</td><td>0.624119</td><td>0.470959</td><td>0.623016</td><td>0.549368</td><td>0.347178</td><td>0.634871</td></tr><tr><th>&vellip;</th><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5 & x6 & x7 & \\\\\n",
       "\t\\hline\n",
       "\t& Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & \\\\\n",
       "\t\\hline\n",
       "\t1 & 0.654616 & 0.719149 & 0.152413 & 0.149021 & 0.906955 & 0.601046 & 0.659532 & $\\dots$ \\\\\n",
       "\t2 & 0.200366 & 0.77317 & 0.54823 & 0.619689 & 0.311176 & 0.348622 & 0.116158 & $\\dots$ \\\\\n",
       "\t3 & 0.0707588 & 0.282814 & 0.809332 & 0.553294 & 0.6751 & 0.187457 & 0.640539 & $\\dots$ \\\\\n",
       "\t4 & 0.263277 & 0.974213 & 0.469272 & 0.976958 & 0.862772 & 0.938512 & 0.617304 & $\\dots$ \\\\\n",
       "\t5 & 0.58163 & 0.949869 & 0.0661904 & 0.863933 & 0.685727 & 0.395609 & 0.311059 & $\\dots$ \\\\\n",
       "\t6 & 0.616957 & 0.809582 & 0.667691 & 0.0241465 & 0.437346 & 0.833641 & 0.11238 & $\\dots$ \\\\\n",
       "\t7 & 0.321904 & 0.569951 & 0.581407 & 0.520908 & 0.34152 & 0.570203 & 0.269444 & $\\dots$ \\\\\n",
       "\t8 & 0.862673 & 0.992583 & 0.153743 & 0.641501 & 0.479036 & 0.858642 & 0.0335735 & $\\dots$ \\\\\n",
       "\t9 & 0.974797 & 0.185455 & 0.183458 & 0.793523 & 0.525708 & 0.263889 & 0.572278 & $\\dots$ \\\\\n",
       "\t10 & 0.573963 & 0.195312 & 0.483336 & 0.268587 & 0.193243 & 0.2797 & 0.265947 & $\\dots$ \\\\\n",
       "\t11 & 0.799476 & 0.509579 & 0.667967 & 0.0499318 & 0.714211 & 0.977774 & 0.588332 & $\\dots$ \\\\\n",
       "\t12 & 0.423771 & 0.280858 & 0.162556 & 0.629407 & 0.41928 & 0.451621 & 0.984615 & $\\dots$ \\\\\n",
       "\t13 & 0.0616925 & 0.611666 & 0.884873 & 0.312924 & 0.303788 & 0.822353 & 0.0804944 & $\\dots$ \\\\\n",
       "\t14 & 0.880249 & 0.256299 & 0.925109 & 0.13784 & 0.499931 & 0.760433 & 0.538864 & $\\dots$ \\\\\n",
       "\t15 & 0.162932 & 0.249386 & 0.763224 & 0.891417 & 0.443553 & 0.771456 & 0.117484 & $\\dots$ \\\\\n",
       "\t16 & 0.882158 & 0.644417 & 0.22389 & 0.781504 & 0.58965 & 0.148809 & 0.46363 & $\\dots$ \\\\\n",
       "\t17 & 0.710762 & 0.299941 & 0.521807 & 0.956344 & 0.412978 & 0.0570247 & 0.435834 & $\\dots$ \\\\\n",
       "\t18 & 0.351941 & 0.591889 & 0.0850421 & 0.619161 & 0.739657 & 0.800522 & 0.91802 & $\\dots$ \\\\\n",
       "\t19 & 0.518158 & 0.870301 & 0.523169 & 0.919283 & 0.780809 & 0.923941 & 0.962397 & $\\dots$ \\\\\n",
       "\t20 & 0.305017 & 0.137517 & 0.0850551 & 0.0845877 & 0.436503 & 0.309409 & 0.620065 & $\\dots$ \\\\\n",
       "\t21 & 0.776885 & 0.151316 & 0.663058 & 0.00522126 & 0.22959 & 0.61607 & 0.815048 & $\\dots$ \\\\\n",
       "\t22 & 0.567911 & 0.318317 & 0.942034 & 0.859492 & 0.882784 & 0.00790537 & 0.870021 & $\\dots$ \\\\\n",
       "\t23 & 0.942238 & 0.563475 & 0.510555 & 0.544423 & 0.714952 & 0.816133 & 0.515208 & $\\dots$ \\\\\n",
       "\t24 & 0.588218 & 0.776325 & 0.356093 & 0.334207 & 0.576845 & 0.788708 & 0.0206834 & $\\dots$ \\\\\n",
       "\t25 & 0.204753 & 0.0273146 & 0.157649 & 0.171963 & 0.891935 & 0.0866493 & 0.0147336 & $\\dots$ \\\\\n",
       "\t26 & 0.968665 & 0.526432 & 0.00336494 & 0.762573 & 0.736109 & 0.599631 & 0.00136439 & $\\dots$ \\\\\n",
       "\t27 & 0.101632 & 0.0437408 & 0.480526 & 0.178961 & 0.614065 & 0.296311 & 0.9397 & $\\dots$ \\\\\n",
       "\t28 & 0.385756 & 0.23059 & 0.0843208 & 0.248933 & 0.888177 & 0.623427 & 0.183932 & $\\dots$ \\\\\n",
       "\t29 & 0.285264 & 0.350805 & 0.494201 & 0.755728 & 0.293184 & 0.36395 & 0.608347 & $\\dots$ \\\\\n",
       "\t30 & 0.724326 & 0.624119 & 0.470959 & 0.623016 & 0.549368 & 0.347178 & 0.634871 & $\\dots$ \\\\\n",
       "\t$\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ &  \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m100×1000 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1        \u001b[0m\u001b[1m x2        \u001b[0m\u001b[1m x3         \u001b[0m\u001b[1m x4        \u001b[0m\u001b[1m x5        \u001b[0m\u001b[1m x6       \u001b[0m\u001b[1m x7   \u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Float64   \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64    \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ 0.654616   0.719149   0.152413    0.149021   0.906955   0.601046  0.659 ⋯\n",
       "   2 │ 0.200366   0.77317    0.54823     0.619689   0.311176   0.348622  0.116\n",
       "   3 │ 0.0707588  0.282814   0.809332    0.553294   0.6751     0.187457  0.640\n",
       "   4 │ 0.263277   0.974213   0.469272    0.976958   0.862772   0.938512  0.617\n",
       "   5 │ 0.58163    0.949869   0.0661904   0.863933   0.685727   0.395609  0.311 ⋯\n",
       "   6 │ 0.616957   0.809582   0.667691    0.0241465  0.437346   0.833641  0.112\n",
       "   7 │ 0.321904   0.569951   0.581407    0.520908   0.34152    0.570203  0.269\n",
       "   8 │ 0.862673   0.992583   0.153743    0.641501   0.479036   0.858642  0.033\n",
       "   9 │ 0.974797   0.185455   0.183458    0.793523   0.525708   0.263889  0.572 ⋯\n",
       "  10 │ 0.573963   0.195312   0.483336    0.268587   0.193243   0.2797    0.265\n",
       "  11 │ 0.799476   0.509579   0.667967    0.0499318  0.714211   0.977774  0.588\n",
       "  ⋮  │     ⋮          ⋮          ⋮           ⋮          ⋮         ⋮            ⋱\n",
       "  91 │ 0.968883   0.241678   0.119772    0.510826   0.186451   0.617466  0.995\n",
       "  92 │ 0.917835   0.3582     0.91722     0.211417   0.537224   0.621612  0.466 ⋯\n",
       "  93 │ 0.789469   0.704847   0.376707    0.671639   0.551814   0.837616  0.308\n",
       "  94 │ 0.879271   0.928887   0.77585     0.752874   0.619996   0.1589    0.000\n",
       "  95 │ 0.817557   0.0886993  0.781368    0.878988   0.548663   0.2104    0.238\n",
       "  96 │ 0.695714   0.447286   0.00368738  0.284621   0.119368   0.210558  0.267 ⋯\n",
       "  97 │ 0.364028   0.95794    0.595653    0.607202   0.472866   0.959469  0.841\n",
       "  98 │ 0.605026   0.0421794  0.558007    0.167992   0.794578   0.02384   0.784\n",
       "  99 │ 0.61941    0.102089   0.150728    0.19859    0.207881   0.429483  0.097\n",
       " 100 │ 0.484436   0.0379625  0.475657    0.0686112  0.97094    0.901547  0.336 ⋯\n",
       "\u001b[31m                                                 994 columns and 79 rows omitted\u001b[0m"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = DataFrame(rand(100, 1000), :auto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  166.701 μs (1986 allocations: 193.33 KiB)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th><th>x6</th><th>x7</th><th>x8</th><th>x9</th></tr><tr><th></th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th></tr></thead><tbody><p>1 rows × 1,000 columns (omitted printing of 991 columns)</p><tr><th>1</th><td>0.654616</td><td>0.719149</td><td>0.152413</td><td>0.149021</td><td>0.906955</td><td>0.601046</td><td>0.659532</td><td>0.49023</td><td>0.414895</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccccccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5 & x6 & x7 & x8 & x9 & \\\\\n",
       "\t\\hline\n",
       "\t& Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & \\\\\n",
       "\t\\hline\n",
       "\t1 & 0.654616 & 0.719149 & 0.152413 & 0.149021 & 0.906955 & 0.601046 & 0.659532 & 0.49023 & 0.414895 & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m1×1000 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1       \u001b[0m\u001b[1m x2       \u001b[0m\u001b[1m x3       \u001b[0m\u001b[1m x4       \u001b[0m\u001b[1m x5       \u001b[0m\u001b[1m x6       \u001b[0m\u001b[1m x7       \u001b[0m\u001b[1m x\u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m F\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ 0.654616  0.719149  0.152413  0.149021  0.906955  0.601046  0.659532  0 ⋯\n",
       "\u001b[31m                                                             993 columns omitted\u001b[0m"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@btime $x[1:1, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  24.823 ns (0 allocations: 0 bytes)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<p>DataFrameRow (1000 columns)</p><table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th><th>x6</th><th>x7</th><th>x8</th><th>x9</th></tr><tr><th></th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th></tr></thead><tbody><tr><th>1</th><td>0.654616</td><td>0.719149</td><td>0.152413</td><td>0.149021</td><td>0.906955</td><td>0.601046</td><td>0.659532</td><td>0.49023</td><td>0.414895</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccccccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5 & x6 & x7 & x8 & x9 & \\\\\n",
       "\t\\hline\n",
       "\t& Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & \\\\\n",
       "\t\\hline\n",
       "\t1 & 0.654616 & 0.719149 & 0.152413 & 0.149021 & 0.906955 & 0.601046 & 0.659532 & 0.49023 & 0.414895 & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1mDataFrameRow\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1       \u001b[0m\u001b[1m x2       \u001b[0m\u001b[1m x3       \u001b[0m\u001b[1m x4       \u001b[0m\u001b[1m x5       \u001b[0m\u001b[1m x6       \u001b[0m\u001b[1m x7       \u001b[0m\u001b[1m x\u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m F\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ 0.654616  0.719149  0.152413  0.149021  0.906955  0.601046  0.659532  0 ⋯\n",
       "\u001b[31m                                                             993 columns omitted\u001b[0m"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@btime $x[1, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  24.823 ns (0 allocations: 0 bytes)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th><th>x6</th><th>x7</th><th>x8</th><th>x9</th></tr><tr><th></th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th></tr></thead><tbody><p>1 rows × 1,000 columns (omitted printing of 991 columns)</p><tr><th>1</th><td>0.654616</td><td>0.719149</td><td>0.152413</td><td>0.149021</td><td>0.906955</td><td>0.601046</td><td>0.659532</td><td>0.49023</td><td>0.414895</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccccccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5 & x6 & x7 & x8 & x9 & \\\\\n",
       "\t\\hline\n",
       "\t& Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & \\\\\n",
       "\t\\hline\n",
       "\t1 & 0.654616 & 0.719149 & 0.152413 & 0.149021 & 0.906955 & 0.601046 & 0.659532 & 0.49023 & 0.414895 & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m1×1000 SubDataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1       \u001b[0m\u001b[1m x2       \u001b[0m\u001b[1m x3       \u001b[0m\u001b[1m x4       \u001b[0m\u001b[1m x5       \u001b[0m\u001b[1m x6       \u001b[0m\u001b[1m x7       \u001b[0m\u001b[1m x\u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m F\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ 0.654616  0.719149  0.152413  0.149021  0.906955  0.601046  0.659532  0 ⋯\n",
       "\u001b[31m                                                             993 columns omitted\u001b[0m"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@btime view($x, 1:1, :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  5.071 μs (39 allocations: 6.17 KiB)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th><th>x6</th><th>x7</th><th>x8</th><th>x9</th></tr><tr><th></th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th></tr></thead><tbody><p>1 rows × 20 columns (omitted printing of 11 columns)</p><tr><th>1</th><td>0.654616</td><td>0.719149</td><td>0.152413</td><td>0.149021</td><td>0.906955</td><td>0.601046</td><td>0.659532</td><td>0.49023</td><td>0.414895</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccccccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5 & x6 & x7 & x8 & x9 & \\\\\n",
       "\t\\hline\n",
       "\t& Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & \\\\\n",
       "\t\\hline\n",
       "\t1 & 0.654616 & 0.719149 & 0.152413 & 0.149021 & 0.906955 & 0.601046 & 0.659532 & 0.49023 & 0.414895 & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m1×20 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1       \u001b[0m\u001b[1m x2       \u001b[0m\u001b[1m x3       \u001b[0m\u001b[1m x4       \u001b[0m\u001b[1m x5       \u001b[0m\u001b[1m x6       \u001b[0m\u001b[1m x7       \u001b[0m\u001b[1m x\u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m F\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ 0.654616  0.719149  0.152413  0.149021  0.906955  0.601046  0.659532  0 ⋯\n",
       "\u001b[31m                                                              13 columns omitted\u001b[0m"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@btime $x[1:1, 1:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  27.939 ns (0 allocations: 0 bytes)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<p>DataFrameRow (20 columns)</p><table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th><th>x6</th><th>x7</th><th>x8</th><th>x9</th></tr><tr><th></th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th></tr></thead><tbody><tr><th>1</th><td>0.654616</td><td>0.719149</td><td>0.152413</td><td>0.149021</td><td>0.906955</td><td>0.601046</td><td>0.659532</td><td>0.49023</td><td>0.414895</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccccccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5 & x6 & x7 & x8 & x9 & \\\\\n",
       "\t\\hline\n",
       "\t& Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & \\\\\n",
       "\t\\hline\n",
       "\t1 & 0.654616 & 0.719149 & 0.152413 & 0.149021 & 0.906955 & 0.601046 & 0.659532 & 0.49023 & 0.414895 & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1mDataFrameRow\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1       \u001b[0m\u001b[1m x2       \u001b[0m\u001b[1m x3       \u001b[0m\u001b[1m x4       \u001b[0m\u001b[1m x5       \u001b[0m\u001b[1m x6       \u001b[0m\u001b[1m x7       \u001b[0m\u001b[1m x\u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m F\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ 0.654616  0.719149  0.152413  0.149021  0.906955  0.601046  0.659532  0 ⋯\n",
       "\u001b[31m                                                              13 columns omitted\u001b[0m"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@btime $x[1, 1:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  26.532 ns (0 allocations: 0 bytes)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th><th>x6</th><th>x7</th><th>x8</th><th>x9</th></tr><tr><th></th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th></tr></thead><tbody><p>1 rows × 20 columns (omitted printing of 11 columns)</p><tr><th>1</th><td>0.654616</td><td>0.719149</td><td>0.152413</td><td>0.149021</td><td>0.906955</td><td>0.601046</td><td>0.659532</td><td>0.49023</td><td>0.414895</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccccccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5 & x6 & x7 & x8 & x9 & \\\\\n",
       "\t\\hline\n",
       "\t& Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & \\\\\n",
       "\t\\hline\n",
       "\t1 & 0.654616 & 0.719149 & 0.152413 & 0.149021 & 0.906955 & 0.601046 & 0.659532 & 0.49023 & 0.414895 & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m1×20 SubDataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1       \u001b[0m\u001b[1m x2       \u001b[0m\u001b[1m x3       \u001b[0m\u001b[1m x4       \u001b[0m\u001b[1m x5       \u001b[0m\u001b[1m x6       \u001b[0m\u001b[1m x7       \u001b[0m\u001b[1m x\u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m F\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ 0.654616  0.719149  0.152413  0.149021  0.906955  0.601046  0.659532  0 ⋯\n",
       "\u001b[31m                                                              13 columns omitted\u001b[0m"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@btime view($x, 1:1, 1:20)"
   ]
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "kernelspec": {
   "display_name": "Julia 1.5.3",
   "language": "julia",
   "name": "julia-1.5"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
